{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Ch `03`: Concept `03`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Regularization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Import the relevant libraries and initialize the hyper-parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "np.random.seed(100)\n",
    "\n",
    "learning_rate = 0.001\n",
    "training_epochs = 1000\n",
    "reg_lambda = 0."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "create a helper method to split the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def split_dataset(x_dataset, y_dataset, ratio):\n",
    "    arr = np.arange(x_dataset.size)\n",
    "    np.random.shuffle(arr)\n",
    "    num_train = int(ratio * x_dataset.size)\n",
    "    x_train = x_dataset[arr[0:num_train]]\n",
    "    y_train = y_dataset[arr[0:num_train]]\n",
    "    x_test = x_dataset[arr[num_train:x_dataset.size]]\n",
    "    y_test = y_dataset[arr[num_train:x_dataset.size]]\n",
    "    return x_train, x_test, y_train, y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Create a fake dataset. y = x^2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "x_dataset = np.linspace(-1, 1, 100)\n",
    "\n",
    "num_coeffs = 9\n",
    "y_dataset_params = [0.] * num_coeffs\n",
    "y_dataset_params[2] = 1\n",
    "y_dataset = 0\n",
    "for i in range(num_coeffs):\n",
    "    y_dataset += y_dataset_params[i] * np.power(x_dataset, i)\n",
    "y_dataset += np.random.randn(*x_dataset.shape) * 0.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Split the dataset into 70% training and testing 30%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "(x_train, x_test, y_train, y_test) = split_dataset(x_dataset, y_dataset, 0.7)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Set up the input/output placeholders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(\"float\")\n",
    "Y = tf.placeholder(\"float\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Define our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def model(X, w):\n",
    "    terms = []\n",
    "    for i in range(num_coeffs):\n",
    "        term = tf.multiply(w[i], tf.pow(X, i))\n",
    "        terms.append(term)\n",
    "    return tf.add_n(terms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Define the regularized cost function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "w = tf.Variable([0.] * num_coeffs, name=\"parameters\")\n",
    "y_model = model(X, w)\n",
    "cost = tf.div(tf.add(tf.reduce_sum(tf.square(Y-y_model)),\n",
    "                     tf.multiply(reg_lambda, tf.reduce_sum(tf.square(w)))),\n",
    "              2*x_train.size)\n",
    "train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Set up the session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Try out various regularization parameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reg lambda 0.0\n",
      "final cost 0.032031\n",
      "reg lambda 0.010101010101\n",
      "final cost 0.0289025\n",
      "reg lambda 0.020202020202\n",
      "final cost 0.0271288\n",
      "reg lambda 0.030303030303\n",
      "final cost 0.0258003\n",
      "reg lambda 0.040404040404\n",
      "final cost 0.024813\n",
      "reg lambda 0.0505050505051\n",
      "final cost 0.0240771\n",
      "reg lambda 0.0606060606061\n",
      "final cost 0.0235204\n",
      "reg lambda 0.0707070707071\n",
      "final cost 0.0230916\n",
      "reg lambda 0.0808080808081\n",
      "final cost 0.022755\n",
      "reg lambda 0.0909090909091\n",
      "final cost 0.0224858\n",
      "reg lambda 0.10101010101\n",
      "final cost 0.0222666\n",
      "reg lambda 0.111111111111\n",
      "final cost 0.0220851\n",
      "reg lambda 0.121212121212\n",
      "final cost 0.0219322\n",
      "reg lambda 0.131313131313\n",
      "final cost 0.0218015\n",
      "reg lambda 0.141414141414\n",
      "final cost 0.0216881\n",
      "reg lambda 0.151515151515\n",
      "final cost 0.0215885\n",
      "reg lambda 0.161616161616\n",
      "final cost 0.0214999\n",
      "reg lambda 0.171717171717\n",
      "final cost 0.0214203\n",
      "reg lambda 0.181818181818\n",
      "final cost 0.0213481\n",
      "reg lambda 0.191919191919\n",
      "final cost 0.0212821\n",
      "reg lambda 0.20202020202\n",
      "final cost 0.0212215\n",
      "reg lambda 0.212121212121\n",
      "final cost 0.0211654\n",
      "reg lambda 0.222222222222\n",
      "final cost 0.0211133\n",
      "reg lambda 0.232323232323\n",
      "final cost 0.0210647\n",
      "reg lambda 0.242424242424\n",
      "final cost 0.0210192\n",
      "reg lambda 0.252525252525\n",
      "final cost 0.0209767\n",
      "reg lambda 0.262626262626\n",
      "final cost 0.0209367\n",
      "reg lambda 0.272727272727\n",
      "final cost 0.0208992\n",
      "reg lambda 0.282828282828\n",
      "final cost 0.0208639\n",
      "reg lambda 0.292929292929\n",
      "final cost 0.0208308\n",
      "reg lambda 0.30303030303\n",
      "final cost 0.0207997\n",
      "reg lambda 0.313131313131\n",
      "final cost 0.0207705\n",
      "reg lambda 0.323232323232\n",
      "final cost 0.0207431\n",
      "reg lambda 0.333333333333\n",
      "final cost 0.0207173\n",
      "reg lambda 0.343434343434\n",
      "final cost 0.0206934\n",
      "reg lambda 0.353535353535\n",
      "final cost 0.0206709\n",
      "reg lambda 0.363636363636\n",
      "final cost 0.0206499\n",
      "reg lambda 0.373737373737\n",
      "final cost 0.0206305\n",
      "reg lambda 0.383838383838\n",
      "final cost 0.0206125\n",
      "reg lambda 0.393939393939\n",
      "final cost 0.0205957\n",
      "reg lambda 0.40404040404\n",
      "final cost 0.0205804\n",
      "reg lambda 0.414141414141\n",
      "final cost 0.0205663\n",
      "reg lambda 0.424242424242\n",
      "final cost 0.0205534\n",
      "reg lambda 0.434343434343\n",
      "final cost 0.0205416\n",
      "reg lambda 0.444444444444\n",
      "final cost 0.0205311\n",
      "reg lambda 0.454545454545\n",
      "final cost 0.0205216\n",
      "reg lambda 0.464646464646\n",
      "final cost 0.0205132\n",
      "reg lambda 0.474747474747\n",
      "final cost 0.0205057\n",
      "reg lambda 0.484848484848\n",
      "final cost 0.0204994\n",
      "reg lambda 0.494949494949\n",
      "final cost 0.020494\n",
      "reg lambda 0.505050505051\n",
      "final cost 0.0204894\n",
      "reg lambda 0.515151515152\n",
      "final cost 0.0204858\n",
      "reg lambda 0.525252525253\n",
      "final cost 0.020483\n",
      "reg lambda 0.535353535354\n",
      "final cost 0.0204811\n",
      "reg lambda 0.545454545455\n",
      "final cost 0.02048\n",
      "reg lambda 0.555555555556\n",
      "final cost 0.0204795\n",
      "reg lambda 0.565656565657\n",
      "final cost 0.0204799\n",
      "reg lambda 0.575757575758\n",
      "final cost 0.020481\n",
      "reg lambda 0.585858585859\n",
      "final cost 0.0204828\n",
      "reg lambda 0.59595959596\n",
      "final cost 0.0204852\n",
      "reg lambda 0.606060606061\n",
      "final cost 0.0204882\n",
      "reg lambda 0.616161616162\n",
      "final cost 0.0204919\n",
      "reg lambda 0.626262626263\n",
      "final cost 0.0204963\n",
      "reg lambda 0.636363636364\n",
      "final cost 0.0205011\n",
      "reg lambda 0.646464646465\n",
      "final cost 0.0205065\n",
      "reg lambda 0.656565656566\n",
      "final cost 0.0205124\n",
      "reg lambda 0.666666666667\n",
      "final cost 0.0205188\n",
      "reg lambda 0.676767676768\n",
      "final cost 0.0205258\n",
      "reg lambda 0.686868686869\n",
      "final cost 0.0205331\n",
      "reg lambda 0.69696969697\n",
      "final cost 0.020541\n",
      "reg lambda 0.707070707071\n",
      "final cost 0.0205492\n",
      "reg lambda 0.717171717172\n",
      "final cost 0.0205578\n",
      "reg lambda 0.727272727273\n",
      "final cost 0.0205668\n",
      "reg lambda 0.737373737374\n",
      "final cost 0.0205764\n",
      "reg lambda 0.747474747475\n",
      "final cost 0.0205861\n",
      "reg lambda 0.757575757576\n",
      "final cost 0.0205963\n",
      "reg lambda 0.767676767677\n",
      "final cost 0.0206067\n",
      "reg lambda 0.777777777778\n",
      "final cost 0.0206174\n",
      "reg lambda 0.787878787879\n",
      "final cost 0.0206286\n",
      "reg lambda 0.79797979798\n",
      "final cost 0.0206399\n",
      "reg lambda 0.808080808081\n",
      "final cost 0.0206516\n",
      "reg lambda 0.818181818182\n",
      "final cost 0.0206635\n",
      "reg lambda 0.828282828283\n",
      "final cost 0.0206756\n",
      "reg lambda 0.838383838384\n",
      "final cost 0.020688\n",
      "reg lambda 0.848484848485\n",
      "final cost 0.0207005\n",
      "reg lambda 0.858585858586\n",
      "final cost 0.0207134\n",
      "reg lambda 0.868686868687\n",
      "final cost 0.0207264\n",
      "reg lambda 0.878787878788\n",
      "final cost 0.0207396\n",
      "reg lambda 0.888888888889\n",
      "final cost 0.0207529\n",
      "reg lambda 0.89898989899\n",
      "final cost 0.0207665\n",
      "reg lambda 0.909090909091\n",
      "final cost 0.0207801\n",
      "reg lambda 0.919191919192\n",
      "final cost 0.020794\n",
      "reg lambda 0.929292929293\n",
      "final cost 0.020808\n",
      "reg lambda 0.939393939394\n",
      "final cost 0.0208222\n",
      "reg lambda 0.949494949495\n",
      "final cost 0.0208364\n",
      "reg lambda 0.959595959596\n",
      "final cost 0.0208508\n",
      "reg lambda 0.969696969697\n",
      "final cost 0.0208652\n",
      "reg lambda 0.979797979798\n",
      "final cost 0.0208798\n",
      "reg lambda 0.989898989899\n",
      "final cost 0.0208943\n",
      "reg lambda 1.0\n",
      "final cost 0.0209091\n"
     ]
    }
   ],
   "source": [
    "for reg_lambda in np.linspace(0,1,100):\n",
    "    for epoch in range(training_epochs):\n",
    "        sess.run(train_op, feed_dict={X: x_train, Y: y_train})\n",
    "    final_cost = sess.run(cost, feed_dict={X: x_test, Y:y_test})\n",
    "    print('reg lambda', reg_lambda)\n",
    "    print('final cost', final_cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Close the session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
